# coding: utf-8

import bs4
import scrapy
import re
from scrapy.contrib.spiders.crawl import CrawlSpider, Rule
from ..items import LejuItem


class LejuSpider(CrawlSpider):
    #
    # 爬取 乐居网站 规则1
    #

    name = 'lejuspider'
    allowed_domains = ['leju.com']
    start_urls = ['http://bj.leju.com/']

    def parse(self, response):
        city_box = [
            ['鞍山', 'http://house.leju.com/as/search/'],
            ['保定', 'http://house.leju.com/bd/search/'],
            ['北海', 'http://house.leju.com/bh/search/'],
            ['北京', 'http://house.leju.com/bj/search/'],
            ['宝鸡', 'http://house.leju.com/aj/search/'],
            ['长春', 'http://house.leju.com/cc/search/'],
            ['长沙', 'http://house.leju.com/cs/search/'],
            ['成都', 'http://house.leju.com/cs/search/'],
            ['重庆', 'http://house.leju.com/cq/search/'],
            ['承德', 'http://house.leju.com/ce/search/'],
            ['赤峰', 'http://house.leju.com/cf/search/'],
            ['滁州', 'http://house.leju.com/cj/search/'],
            ['大连', 'http://house.leju.com/dl/search/'],
            ['恩施', 'http://house.leju.com/es/search/'],
            ['佛山', 'http://house.leju.com/fs/search/'],
            ['沈阳', 'http://house.leju.com/sy/search/'],
            ['福州', 'http://house.leju.com/fz/search/'],
            ['桂林', 'http://house.leju.com/gl/search/'],
            ['贵阳', 'http://house.leju.com/gy/search/'],
            ['海南', 'http://house.leju.com/hi/search/'],
            ['杭州', 'http://house.leju.com/hz/search/'],
            ['合肥', 'http://house.leju.com/hf/search/'],
            ['呼和浩特', 'http://house.leju.com/hu/search/'],
            ['邯郸', 'http://house.leju.com/hd/search/'],
            ['汉中', 'http://house.leju.com/az/search/'],
            ['济南', 'http://house.leju.com/sd/search/'],
            ['金华', 'http://house.leju.com/jh/search/'],
            ['晋城', 'http://house.leju.com/jc/search/'],
            ['昆山', 'http://house.leju.com/ks/search/'],
            ['兰州', 'http://house.leju.com/lz/search/'],
            ['洛阳', 'http://house.leju.com/ly/search/'],
            ['柳州', 'http://house.leju.com/la/search/'],
            ['娄底', 'http://house.leju.com/ld/search/'],
            ['龙岩', 'http://house.leju.com/lo/search/'],
            ['临沂', 'http://house.leju.com/lb/search/'],
            ['丽水', 'http://house.leju.com/lk/search/'],
            ['眉山', 'http://house.leju.com/ms/search/'],
            ['绵阳', 'http://house.leju.com/my/search/'],
            ['梅州', 'http://house.leju.com/mz/search/'],
            ['南昌', 'http://house.leju.com/nc/search/'],
            ['南通', 'http://house.leju.com/nt/search/'],
            ['宁波', 'http://house.leju.com/nb/search/'],
            ['南阳', 'http://house.leju.com/ny/search/'],
            ['秦皇岛', 'http://house.leju.com/qi/search/'],
            ['三亚', 'http://house.leju.com/sa/search/'],
            ['上海', 'http://house.leju.com/sh/search/'],
            ['深圳', 'http://house.leju.com/sz/search/'],
            ['苏州', 'http://house.leju.com/su/search/'],
            ['汕头', 'http://house.leju.com/st/search/'],
            ['邵阳', 'http://house.leju.com/sb/search/'],
            ['太原', 'http://house.leju.com/ty/search/'],
            ['唐山', 'http://house.leju.com/ts/search/'],
            ['南京', 'http://house.leju.com/nj/search/'],
            ['威海', 'http://house.leju.com/we/search/'],
            ['武汉', 'http://house.leju.com/wh/search/'],
            ['芜湖', 'http://house.leju.com/wu/search/'],
            ['无锡', 'http://house.leju.com/wx/search/'],
            ['厦门', 'http://house.leju.com/xm/search/'],
            ['西安', 'http://house.leju.com/sx/search/'],
            ['徐州', 'http://house.leju.com/xz/search/'],
            ['襄阳', 'http://house.leju.com/xf/search/'],
            ['孝感', 'http://house.leju.com/xh/search/'],
            ['烟台', 'http://house.leju.com/yt/search/'],
            ['银川', 'http://house.leju.com/yi/search/'],
            ['云南', 'http://house.leju.com/yn/search/'],
            ['益阳', 'http://house.leju.com/yy/search/'],
            ['宜兴', 'http://house.leju.com/yx/search/'],
            ['盐城', 'http://house.leju.com/ya/search/'],
            ['宜昌', 'http://house.leju.com/yc/search/'],
            ['永州', 'http://house.leju.com/yz/search/'],
            ['郑州', 'http://house.leju.com/hn/search/'],
            ['中山', 'http://house.leju.com/zs/search/'],
            ['珠海', 'http://house.leju.com/zh/search/'],
            ['遵义', 'http://house.leju.com/zy/search/']]

        #
        # 获取乐居网 部分城市
        #

        for child in city_box:
            city = child[0]
            city_url = child[1]
            city_id = city_url.replace('http://house.leju.com/', '').replace('/search/', '')
            citys = {
                'website': '乐居网', 'web_url': 'leju.com',
                'city': city, 'city_id': city_id
            }
            yield scrapy.Request(city_url, callback=self.parse_cityPage_url, meta=citys)

    def parse_cityPage_url(self, response):
        meta = response.meta
        data = response.body
        city_id = meta['city_id']
        soup = bs4.BeautifulSoup(data, 'lxml')

        #
        # 获取城市总页数 进行翻页
        #

        pages = soup.select('.b_pages > a ')[-1].get('href')
        page_num = pages.replace('/' + city_id + '/search/?page=', '').replace(
                '#wt_source=pc_search_down_fy', '').replace('http://house.leju.com', '')
        num = int(page_num.strip())
        for n in range(num):
            site_url = 'http://house.leju.com/' + city_id + '/search/?page=' + str(
                    n + 1) + '#wt_source=pc_search_down_fy'

            yield scrapy.Request(site_url, callback=self.parse_city_estate, meta=meta)

    def parse_city_estate(self, response):
        meta = response.meta
        city_id = meta['city_id']
        data = response.body
        soup = bs4.BeautifulSoup(data, 'lxml')
        boxs = soup.find_all('div', class_='b_titBox')

        #
        # 获取 楼盘信息
        #

        for child in boxs:
            cols = child.find_all('h2')
            for col in cols:
                estate = col.a.get_text()
                estate_url = col.a.get('href')
                estate_id = col.a.get('href').replace('http://house.leju.com/', '').replace(
                        '/#wt_source=pc_search_lpxx_bt', '').replace('/', '')
                estate_id2 = estate_id.replace(city_id, '')
                areas = col.find_next_siblings('h3')[0]
                area = re.findall(r'\[.+\]', areas.get_text())
                area = area[0].replace('[', '').replace(']', '')
                item = LejuItem()
                item['website'] = meta['website']
                item['web_url'] = meta['web_url']
                item['city'] = meta['city']
                item['city_id'] = meta['city_id']
                item['area'] = area
                item['estate'] = estate
                item['estate_id'] = estate_id
                item['estate_id2'] = estate_id2
                item['estate_url'] = estate_url
                yield item
